{
    class Rl {
        constructor (o) {
            this._o = o;
            this.qtable = o.qtable;
            this.env = o.env;
        }
        
        start(callback) {
            let x = 0;
            let isTerminated = false;
            let state = 0;
            let action = "left";
            let _state = 0;
            let reward = 0;
            let qPredict = 0;  
            let qTarget = 0;

            let roundCounter = 0;
            let stepCounter = 0;
            let stepArr = [];

            this.env.update(state);

            let timer = setInterval(() => {
                if ( isTerminated ) {
                    window.clearInterval(timer);
                    callback(this.qtable.getTable(), roundCounter, stepArr);
                    return false;
                } else {
                    action = this._chooseAction(state);
                    let envFeedback = this._getEnvFeedback(state, action);
                    _state = envFeedback._state;
                    reward = envFeedback.reward;
                    qPredict = this.qtable.q(state, action).value;
                    if ( _state ===  "__terminal__" ) {
                        qTarget = reward; // next state is terminal
                        _state = 0;
                        stepArr.push(stepCounter);
                        stepCounter = 0;
                        roundCounter++;
                    } else {
                        stepCounter++;
                        qTarget = reward + this._o.gamma * this.qtable.max({state: _state,action: "__require__"}).value;
                    }
                    let oldVal = this.qtable.q(state, action).value;
                    this.qtable.q(state, action).set( oldVal + this._o.alpha * (qTarget - qPredict) ) // update
                    state = _state;
                    this.env.update(state);

                    //function convergence
                    if ( stepArr.length > 2 ) {
                        if ( stepArr[stepArr.length - 1] === stepArr[stepArr.length - 2] ) {
                            isTerminated = true;
                        }
                    }
                }
            }, 100);
        }

        _chooseAction(state) {
            let maxActionName = null;
            let maxEle = this.qtable.max({
                state: state,
                action: "__require__",
            });
            // This is how to choose an action
            // act non-greedy or state-action have no value
            if ( Math.random() > this._o.epsilon || maxEle.value === 0 ) {
                maxActionName = this.qtable.random("actions");
            } else { // act greedy 
                maxActionName = maxEle.name 
            }
            return maxActionName;
        }

        _getEnvFeedback(state, action) {
            let _state = "";
            let reward = "";
            // This is how agent will interact with the environment
            if ( action == 'right' ) {    // move right
                if ( state == this.qtable.size.y - 2 ) { // terminate
                    _state = '__terminal__';
                    reward = 1;
                } else {
                    _state = state + 1;
                    reward = 0
                }
            } else { // move left
                reward = 0
                if ( state == 0 ) { 
                    _state = state  // reach the wall
                } else {
                    _state = state - 1
                }
            }
            return { _state, reward };
        }

    }

    window.Rl = Rl;
}